df2.to_pickle('df2.pkl')
sentence_splitter = spacy.blank("en") sentence_splitter.add_pipe("sentencizer") docs_with_metadata = ( (row.text_clean, (row.Index, row.date)) for row in df2[["text_clean", "date"]].itertuples() ) sents = pd.DataFrame( {"text": s.orth_, "review_id": id, "date": date} for doc, (id, date) in sentence_splitter.pipe(docs_with_metadata, as_tuples=True) for s in doc.sents )